


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.cuda &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/cuda.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" /> 

  
  <script src="../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/cuda.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../index.html">Module code</a> &gt;</li>
        
          <li><a href="../torch.html">torch</a> &gt;</li>
        
      <li>torch.cuda</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.cuda</h1><div class="highlight"><pre>
<span></span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">This package adds support for CUDA tensor types, that implement the same</span>
<span class="sd">function as CPU tensors, but they utilize GPUs for computation.</span>

<span class="sd">It is lazily initialized, so you can always import it, and use</span>
<span class="sd">:func:`is_available()` to determine if your system supports CUDA.</span>

<span class="sd">:ref:`cuda-semantics` has more details about working with CUDA.</span>
<span class="sd">&quot;&quot;&quot;</span>

<span class="kn">import</span> <span class="nn">contextlib</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">traceback</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">threading</span>
<span class="kn">from</span> <span class="nn">torch._six</span> <span class="kn">import</span> <span class="n">raise_from</span>
<span class="kn">from</span> <span class="nn">._utils</span> <span class="kn">import</span> <span class="n">_get_device_index</span>
<span class="kn">import</span> <span class="nn">torch._C</span>

<span class="k">try</span><span class="p">:</span>
    <span class="kn">from</span> <span class="nn">torch._C</span> <span class="kn">import</span> <span class="n">_cudart</span>
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
    <span class="n">_cudart</span> <span class="o">=</span> <span class="kc">None</span>

<span class="n">_initialized</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">_tls</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">local</span><span class="p">()</span>
<span class="n">_initialization_lock</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>
<span class="n">_queued_calls</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c1"># don&#39;t invoke these until initialization occurs</span>
<span class="n">_is_in_bad_fork</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="p">,</span> <span class="s2">&quot;_cuda_isInBadFork&quot;</span><span class="p">,</span> <span class="k">lambda</span><span class="p">:</span> <span class="kc">False</span><span class="p">)</span>


<div class="viewcode-block" id="is_available"><a class="viewcode-back" href="../../cuda.html#torch.cuda.is_available">[docs]</a><span class="k">def</span> <span class="nf">is_available</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns a bool indicating if CUDA is currently available.&quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="p">,</span> <span class="s1">&#39;_cuda_isDriverSufficient&#39;</span><span class="p">)</span> <span class="ow">or</span>
            <span class="ow">not</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_isDriverSufficient</span><span class="p">()):</span>
        <span class="k">return</span> <span class="kc">False</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDeviceCount</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span></div>


<span class="k">def</span> <span class="nf">_sleep</span><span class="p">(</span><span class="n">cycles</span><span class="p">):</span>
    <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_sleep</span><span class="p">(</span><span class="n">cycles</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">_check_driver</span><span class="p">():</span>
    <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="p">,</span> <span class="s1">&#39;_cuda_isDriverSufficient&#39;</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">AssertionError</span><span class="p">(</span><span class="s2">&quot;Torch not compiled with CUDA enabled&quot;</span><span class="p">)</span>
    <span class="k">if</span> <span class="ow">not</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_isDriverSufficient</span><span class="p">():</span>
        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDriverVersion</span><span class="p">()</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="c1"># found no NVIDIA driver on the system</span>
            <span class="k">raise</span> <span class="ne">AssertionError</span><span class="p">(</span><span class="s2">&quot;&quot;&quot;</span>
<span class="s2">Found no NVIDIA driver on your system. Please check that you</span>
<span class="s2">have an NVIDIA GPU and installed a driver from</span>
<span class="s2">http://www.nvidia.com/Download/index.aspx&quot;&quot;&quot;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># TODO: directly link to the alternative bin that needs install</span>
            <span class="k">raise</span> <span class="ne">AssertionError</span><span class="p">(</span><span class="s2">&quot;&quot;&quot;</span>
<span class="s2">The NVIDIA driver on your system is too old (found version </span><span class="si">{}</span><span class="s2">).</span>
<span class="s2">Please update your GPU driver by downloading and installing a new</span>
<span class="s2">version from the URL: http://www.nvidia.com/Download/index.aspx</span>
<span class="s2">Alternatively, go to: https://pytorch.org to install</span>
<span class="s2">a PyTorch version that has been compiled with your version</span>
<span class="s2">of the CUDA driver.&quot;&quot;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDriverVersion</span><span class="p">())))</span>


<span class="k">def</span> <span class="nf">_check_capability</span><span class="p">():</span>
    <span class="n">incorrect_binary_warn</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
<span class="s2">    Found GPU</span><span class="si">%d</span><span class="s2"> </span><span class="si">%s</span><span class="s2"> which requires CUDA_VERSION &gt;= </span><span class="si">%d</span><span class="s2"> to</span>
<span class="s2">     work properly, but your PyTorch was compiled</span>
<span class="s2">     with CUDA_VERSION </span><span class="si">%d</span><span class="s2">. Please install the correct PyTorch binary</span>
<span class="s2">     using instructions from https://pytorch.org</span>
<span class="s2">    &quot;&quot;&quot;</span>

    <span class="n">old_gpu_warn</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
<span class="s2">    Found GPU</span><span class="si">%d</span><span class="s2"> </span><span class="si">%s</span><span class="s2"> which is of cuda capability </span><span class="si">%d</span><span class="s2">.</span><span class="si">%d</span><span class="s2">.</span>
<span class="s2">    PyTorch no longer supports this GPU because it is too old.</span>
<span class="s2">    The minimum cuda capability that we support is 3.5.</span>
<span class="s2">    &quot;&quot;&quot;</span>

    <span class="n">CUDA_VERSION</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getCompiledVersion</span><span class="p">()</span>
    <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">device_count</span><span class="p">()):</span>
        <span class="n">capability</span> <span class="o">=</span> <span class="n">get_device_capability</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
        <span class="n">major</span> <span class="o">=</span> <span class="n">capability</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="n">minor</span> <span class="o">=</span> <span class="n">capability</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">name</span> <span class="o">=</span> <span class="n">get_device_name</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">capability</span> <span class="o">==</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">or</span> <span class="n">major</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">old_gpu_warn</span> <span class="o">%</span> <span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">major</span><span class="p">,</span> <span class="n">capability</span><span class="p">[</span><span class="mi">1</span><span class="p">]))</span>
        <span class="k">elif</span> <span class="n">CUDA_VERSION</span> <span class="o">&lt;=</span> <span class="mi">9000</span> <span class="ow">and</span> <span class="n">major</span> <span class="o">&gt;=</span> <span class="mi">7</span> <span class="ow">and</span> <span class="n">minor</span> <span class="o">&gt;=</span> <span class="mi">5</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">incorrect_binary_warn</span> <span class="o">%</span> <span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="mi">10000</span><span class="p">,</span> <span class="n">CUDA_VERSION</span><span class="p">))</span>


<div class="viewcode-block" id="is_initialized"><a class="viewcode-back" href="../../cuda.html#torch.cuda.is_initialized">[docs]</a><span class="k">def</span> <span class="nf">is_initialized</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns whether PyTorch&#39;s CUDA state has been initialized.&quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="n">_initialized</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">_is_in_bad_fork</span><span class="p">()</span></div>


<span class="k">def</span> <span class="nf">_lazy_call</span><span class="p">(</span><span class="n">callable</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">is_initialized</span><span class="p">():</span>
        <span class="n">callable</span><span class="p">()</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="c1"># Don&#39;t store the actual traceback to avoid memory cycle</span>
        <span class="n">_queued_calls</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">callable</span><span class="p">,</span> <span class="n">traceback</span><span class="o">.</span><span class="n">format_stack</span><span class="p">()))</span>

<span class="n">_lazy_call</span><span class="p">(</span><span class="n">_check_capability</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">DeferredCudaCallError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
    <span class="k">pass</span>


<div class="viewcode-block" id="init"><a class="viewcode-back" href="../../cuda.html#torch.cuda.init">[docs]</a><span class="k">def</span> <span class="nf">init</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Initialize PyTorch&#39;s CUDA state.  You may need to call</span>
<span class="sd">    this explicitly if you are interacting with PyTorch via</span>
<span class="sd">    its C API, as Python bindings for CUDA functionality will not</span>
<span class="sd">    be until this initialization takes place.  Ordinary users</span>
<span class="sd">    should not need this, as all of PyTorch&#39;s CUDA methods</span>
<span class="sd">    automatically initialize CUDA state on-demand.</span>

<span class="sd">    Does nothing if the CUDA state is already initialized.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span></div>


<span class="k">def</span> <span class="nf">_lazy_init</span><span class="p">():</span>
    <span class="k">global</span> <span class="n">_initialized</span><span class="p">,</span> <span class="n">_queued_calls</span>
    <span class="k">if</span> <span class="n">is_initialized</span><span class="p">()</span> <span class="ow">or</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">_tls</span><span class="p">,</span> <span class="s1">&#39;is_initializing&#39;</span><span class="p">):</span>
        <span class="k">return</span>
    <span class="k">with</span> <span class="n">_initialization_lock</span><span class="p">:</span>
        <span class="c1"># We be double-checked locking, boys!  This is OK because</span>
        <span class="c1"># the above test was GIL protected anyway.  The inner test</span>
        <span class="c1"># is for when a thread blocked on some other thread which was</span>
        <span class="c1"># doing the initialization; when they get the lock, they will</span>
        <span class="c1"># find there is nothing left to do.</span>
        <span class="k">if</span> <span class="n">is_initialized</span><span class="p">():</span>
            <span class="k">return</span>
        <span class="c1"># It is important to prevent other threads from entering _lazy_init</span>
        <span class="c1"># immediately, while we are still guaranteed to have the GIL, because some</span>
        <span class="c1"># of the C calls we make below will release the GIL</span>
        <span class="k">if</span> <span class="n">_is_in_bad_fork</span><span class="p">():</span>
            <span class="kn">from</span> <span class="nn">sys</span> <span class="kn">import</span> <span class="n">version_info</span>
            <span class="k">if</span> <span class="n">version_info</span> <span class="o">&lt;</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">):</span>
                <span class="n">msg</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;To use CUDA with multiprocessing, you must use Python &quot;</span>
                       <span class="s2">&quot;3.4+ and the &#39;spawn&#39; start method&quot;</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">msg</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;To use CUDA with multiprocessing, you must use the &quot;</span>
                       <span class="s2">&quot;&#39;spawn&#39; start method&quot;</span><span class="p">)</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                <span class="s2">&quot;Cannot re-initialize CUDA in forked subprocess. &quot;</span> <span class="o">+</span> <span class="n">msg</span><span class="p">)</span>
        <span class="n">_check_driver</span><span class="p">()</span>
        <span class="k">if</span> <span class="n">_cudart</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">AssertionError</span><span class="p">(</span>
                <span class="s2">&quot;libcudart functions unavailable. It looks like you have a broken build?&quot;</span><span class="p">)</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_init</span><span class="p">()</span>
        <span class="c1"># Some of the queued calls may reentrantly call _lazy_init();</span>
        <span class="c1"># we need to just return without initializing in that case.</span>
        <span class="c1"># However, we must not let any *other* threads in!</span>
        <span class="n">_tls</span><span class="o">.</span><span class="n">is_initializing</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">queued_call</span><span class="p">,</span> <span class="n">orig_traceback</span> <span class="ow">in</span> <span class="n">_queued_calls</span><span class="p">:</span>
                <span class="k">try</span><span class="p">:</span>
                    <span class="n">queued_call</span><span class="p">()</span>
                <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
                    <span class="n">msg</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;CUDA call failed lazily at initialization with error: </span><span class="si">{}</span><span class="se">\n\n</span><span class="s2">&quot;</span>
                           <span class="s2">&quot;CUDA call was originally invoked at:</span><span class="se">\n\n</span><span class="si">{}</span><span class="s2">&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">),</span> <span class="n">orig_traceback</span><span class="p">)</span>
                    <span class="n">raise_from</span><span class="p">(</span><span class="n">DeferredCudaCallError</span><span class="p">(</span><span class="n">msg</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span>
        <span class="k">finally</span><span class="p">:</span>
            <span class="nb">delattr</span><span class="p">(</span><span class="n">_tls</span><span class="p">,</span> <span class="s1">&#39;is_initializing&#39;</span><span class="p">)</span>
        <span class="n">_initialized</span> <span class="o">=</span> <span class="kc">True</span>


<span class="k">def</span> <span class="nf">cudart</span><span class="p">():</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">_cudart</span>


<span class="k">class</span> <span class="nc">cudaStatus</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="n">SUCCESS</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="n">ERROR_NOT_READY</span> <span class="o">=</span> <span class="mi">34</span>


<span class="k">class</span> <span class="nc">CudaError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">code</span><span class="p">):</span>
        <span class="n">msg</span> <span class="o">=</span> <span class="n">_cudart</span><span class="o">.</span><span class="n">cudaGetErrorString</span><span class="p">(</span><span class="n">code</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">CudaError</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1"> (</span><span class="si">{1}</span><span class="s1">)&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="n">code</span><span class="p">))</span>


<span class="k">def</span> <span class="nf">check_error</span><span class="p">(</span><span class="n">res</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">res</span> <span class="o">!=</span> <span class="n">_cudart</span><span class="o">.</span><span class="n">cudaError</span><span class="o">.</span><span class="n">success</span><span class="p">:</span>
        <span class="k">raise</span> <span class="n">CudaError</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>


<div class="viewcode-block" id="device"><a class="viewcode-back" href="../../cuda.html#torch.cuda.device">[docs]</a><span class="k">class</span> <span class="nc">device</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Context-manager that changes the selected device.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int): device index to select. It&#39;s a no-op if</span>
<span class="sd">            this argument is a negative integer or ``None``.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">device</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">idx</span> <span class="o">=</span> <span class="n">_get_device_index</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">prev_idx</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>

    <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">idx</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
            <span class="k">return</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">prev_idx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDevice</span><span class="p">()</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">prev_idx</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">idx</span><span class="p">:</span>
            <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setDevice</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">idx</span><span class="p">)</span>
        <span class="n">_lazy_init</span><span class="p">()</span>

    <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">prev_idx</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">idx</span><span class="p">:</span>
            <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setDevice</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">prev_idx</span><span class="p">)</span>
        <span class="k">return</span> <span class="kc">False</span></div>


<div class="viewcode-block" id="device_of"><a class="viewcode-back" href="../../cuda.html#torch.cuda.device_of">[docs]</a><span class="k">class</span> <span class="nc">device_of</span><span class="p">(</span><span class="n">device</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Context-manager that changes the current device to that of given object.</span>

<span class="sd">    You can use both tensors and storages as arguments. If a given object is</span>
<span class="sd">    not allocated on a GPU, this is a no-op.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        obj (Tensor or Storage): object allocated on the selected device.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
        <span class="n">idx</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">get_device</span><span class="p">()</span> <span class="k">if</span> <span class="n">obj</span><span class="o">.</span><span class="n">is_cuda</span> <span class="k">else</span> <span class="o">-</span><span class="mi">1</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">device_of</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span></div>


<div class="viewcode-block" id="set_device"><a class="viewcode-back" href="../../cuda.html#torch.cuda.set_device">[docs]</a><span class="k">def</span> <span class="nf">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the current device.</span>

<span class="sd">    Usage of this function is discouraged in favor of :any:`device`. In most</span>
<span class="sd">    cases it&#39;s better to use ``CUDA_VISIBLE_DEVICES`` environmental variable.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int): selected device. This function is a no-op</span>
<span class="sd">            if this argument is negative.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">device</span> <span class="o">=</span> <span class="n">_get_device_index</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">device</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">:</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setDevice</span><span class="p">(</span><span class="n">device</span><span class="p">)</span></div>


<div class="viewcode-block" id="get_device_name"><a class="viewcode-back" href="../../cuda.html#torch.cuda.get_device_name">[docs]</a><span class="k">def</span> <span class="nf">get_device_name</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Gets the name of a device.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int, optional): device for which to return the</span>
<span class="sd">            name. This function is a no-op if this argument is a negative</span>
<span class="sd">            integer. It uses the current device, given by :func:`~torch.cuda.current_device`,</span>
<span class="sd">            if :attr:`device` is ``None`` (default).</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="n">get_device_properties</span><span class="p">(</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">name</span></div>


<div class="viewcode-block" id="get_device_capability"><a class="viewcode-back" href="../../cuda.html#torch.cuda.get_device_capability">[docs]</a><span class="k">def</span> <span class="nf">get_device_capability</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Gets the cuda capability of a device.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int, optional): device for which to return the</span>
<span class="sd">            device capability. This function is a no-op if this argument is</span>
<span class="sd">            a negative integer. It uses the current device, given by</span>
<span class="sd">            :func:`~torch.cuda.current_device`, if :attr:`device` is ``None``</span>
<span class="sd">            (default).</span>

<span class="sd">    Returns:</span>
<span class="sd">        tuple(int, int): the major and minor cuda capability of the device</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">prop</span> <span class="o">=</span> <span class="n">get_device_properties</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">prop</span><span class="o">.</span><span class="n">major</span><span class="p">,</span> <span class="n">prop</span><span class="o">.</span><span class="n">minor</span></div>


<span class="k">def</span> <span class="nf">get_device_properties</span><span class="p">(</span><span class="n">device</span><span class="p">):</span>
    <span class="n">_lazy_init</span><span class="p">()</span>  <span class="c1"># will define _get_device_properties and _CudaDeviceProperties</span>
    <span class="n">device</span> <span class="o">=</span> <span class="n">_get_device_index</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">device</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">device</span> <span class="o">&gt;=</span> <span class="n">device_count</span><span class="p">():</span>
        <span class="k">raise</span> <span class="ne">AssertionError</span><span class="p">(</span><span class="s2">&quot;Invalid device id&quot;</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">_get_device_properties</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>


<div class="viewcode-block" id="stream"><a class="viewcode-back" href="../../cuda.html#torch.cuda.stream">[docs]</a><span class="nd">@contextlib</span><span class="o">.</span><span class="n">contextmanager</span>
<span class="k">def</span> <span class="nf">stream</span><span class="p">(</span><span class="n">stream</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Context-manager that selects a given stream.</span>

<span class="sd">    All CUDA kernels queued within its context will be enqueued on a selected</span>
<span class="sd">    stream.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        stream (Stream): selected stream. This manager is a no-op if it&#39;s</span>
<span class="sd">            ``None``.</span>

<span class="sd">    .. note:: Streams are per-device. If the selected stream is not on the</span>
<span class="sd">        current device, this function will also change the current device to</span>
<span class="sd">        match the stream.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">stream</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">yield</span>
        <span class="k">return</span>
    <span class="n">src_prev_stream</span> <span class="o">=</span> <span class="n">current_stream</span><span class="p">()</span>

    <span class="k">if</span> <span class="n">src_prev_stream</span><span class="o">.</span><span class="n">device</span> <span class="o">!=</span> <span class="n">stream</span><span class="o">.</span><span class="n">device</span><span class="p">:</span>
        <span class="c1"># The given stream is on a different device; have to restore the</span>
        <span class="c1"># current_stream on that device on exit as well</span>
        <span class="k">with</span> <span class="n">device</span><span class="p">(</span><span class="n">stream</span><span class="o">.</span><span class="n">device</span><span class="p">):</span>
            <span class="n">dst_prev_stream</span> <span class="o">=</span> <span class="n">current_stream</span><span class="p">()</span>

    <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setStream</span><span class="p">(</span><span class="n">stream</span><span class="o">.</span><span class="n">_cdata</span><span class="p">)</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="k">yield</span>
    <span class="k">finally</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">src_prev_stream</span><span class="o">.</span><span class="n">device</span> <span class="o">!=</span> <span class="n">stream</span><span class="o">.</span><span class="n">device</span><span class="p">:</span>
            <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setStream</span><span class="p">(</span><span class="n">dst_prev_stream</span><span class="o">.</span><span class="n">_cdata</span><span class="p">)</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_setStream</span><span class="p">(</span><span class="n">src_prev_stream</span><span class="o">.</span><span class="n">_cdata</span><span class="p">)</span></div>


<div class="viewcode-block" id="device_count"><a class="viewcode-back" href="../../cuda.html#torch.cuda.device_count">[docs]</a><span class="k">def</span> <span class="nf">device_count</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the number of GPUs available.&quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">is_available</span><span class="p">():</span>
        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDeviceCount</span><span class="p">()</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="k">return</span> <span class="mi">0</span></div>


<div class="viewcode-block" id="current_device"><a class="viewcode-back" href="../../cuda.html#torch.cuda.current_device">[docs]</a><span class="k">def</span> <span class="nf">current_device</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the index of a currently selected device.&quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDevice</span><span class="p">()</span></div>


<div class="viewcode-block" id="synchronize"><a class="viewcode-back" href="../../cuda.html#torch.cuda.synchronize">[docs]</a><span class="k">def</span> <span class="nf">synchronize</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Waits for all kernels in all streams on a CUDA device to complete.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int, optional): device for which to synchronize.</span>
<span class="sd">            It uses the current device, given by :func:`~torch.cuda.current_device`,</span>
<span class="sd">            if :attr:`device` is ``None`` (default).</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="n">device</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_synchronize</span><span class="p">()</span></div>


<div class="viewcode-block" id="ipc_collect"><a class="viewcode-back" href="../../cuda.html#torch.cuda.ipc_collect">[docs]</a><span class="k">def</span> <span class="nf">ipc_collect</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Force collects GPU memory after it has been released by CUDA IPC.</span>

<span class="sd">    .. note::</span>
<span class="sd">        Checks if any sent CUDA tensors could be cleaned from the memory. Force</span>
<span class="sd">        closes shared memory file used for reference counting if there is no</span>
<span class="sd">        active counters. Useful when the producer process stopped actively sending</span>
<span class="sd">        tensors and want to release unused memory.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_ipc_collect</span><span class="p">()</span></div>


<div class="viewcode-block" id="current_stream"><a class="viewcode-back" href="../../cuda.html#torch.cuda.current_stream">[docs]</a><span class="k">def</span> <span class="nf">current_stream</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the currently selected :class:`Stream` for a given device.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int, optional): selected device. Returns</span>
<span class="sd">            the currently selected :class:`Stream` for the current device, given</span>
<span class="sd">            by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None``</span>
<span class="sd">            (default).</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">Stream</span><span class="p">(</span><span class="n">_cdata</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getCurrentStream</span><span class="p">(</span>
        <span class="n">_get_device_index</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span></div>


<div class="viewcode-block" id="default_stream"><a class="viewcode-back" href="../../cuda.html#torch.cuda.default_stream">[docs]</a><span class="k">def</span> <span class="nf">default_stream</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the default :class:`Stream` for a given device.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        device (torch.device or int, optional): selected device. Returns</span>
<span class="sd">            the default :class:`Stream` for the current device, given by</span>
<span class="sd">            :func:`~torch.cuda.current_device`, if :attr:`device` is ``None``</span>
<span class="sd">            (default).</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">Stream</span><span class="p">(</span><span class="n">_cdata</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getDefaultStream</span><span class="p">(</span>
        <span class="n">_get_device_index</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span></div>


<div class="viewcode-block" id="current_blas_handle"><a class="viewcode-back" href="../../cuda.html#torch.cuda.current_blas_handle">[docs]</a><span class="k">def</span> <span class="nf">current_blas_handle</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns cublasHandle_t pointer to current cuBLAS handle&quot;&quot;&quot;</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_cuda_getCurrentBlasHandle</span><span class="p">()</span></div>


<span class="kn">from</span> <span class="nn">.memory</span> <span class="kn">import</span> <span class="o">*</span>


<span class="kn">from</span> <span class="nn">.random</span> <span class="kn">import</span> <span class="o">*</span>

<span class="c1">################################################################################</span>
<span class="c1"># Define Storage and Tensor classes</span>
<span class="c1">################################################################################</span>


<span class="kn">from</span> <span class="nn">..storage</span> <span class="kn">import</span> <span class="n">_StorageBase</span>


<span class="k">def</span> <span class="nf">_dummy_type</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">init_err</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">class_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
            <span class="s2">&quot;Tried to instantiate dummy base class </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">class_name</span><span class="p">))</span>
    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="n">storage_name</span><span class="p">,</span> <span class="p">(</span><span class="nb">object</span><span class="p">,),</span> <span class="p">{</span><span class="s2">&quot;__init__&quot;</span><span class="p">:</span> <span class="n">init_err</span><span class="p">})</span>


<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="p">,</span> <span class="s1">&#39;CudaDoubleStorageBase&#39;</span><span class="p">):</span>
    <span class="c1"># Define dummy base classes</span>
    <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;Double&#39;</span><span class="p">,</span> <span class="s1">&#39;Float&#39;</span><span class="p">,</span> <span class="s1">&#39;Long&#39;</span><span class="p">,</span> <span class="s1">&#39;Int&#39;</span><span class="p">,</span> <span class="s1">&#39;Short&#39;</span><span class="p">,</span> <span class="s1">&#39;Char&#39;</span><span class="p">,</span> <span class="s1">&#39;Byte&#39;</span><span class="p">,</span> <span class="s1">&#39;Half&#39;</span><span class="p">,</span> <span class="s1">&#39;Bool&#39;</span><span class="p">,</span> <span class="s1">&#39;BFloat16&#39;</span><span class="p">]:</span>
        <span class="n">storage_name</span> <span class="o">=</span> <span class="s1">&#39;Cuda</span><span class="si">{0}</span><span class="s1">StorageBase&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
        <span class="n">tensor_name</span> <span class="o">=</span> <span class="s1">&#39;Cuda</span><span class="si">{0}</span><span class="s1">TensorBase&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>

        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">storage_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_dummy_type</span><span class="p">(</span><span class="n">storage_name</span><span class="p">)</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">tensor_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_dummy_type</span><span class="p">(</span><span class="n">tensor_name</span><span class="p">)</span>

    <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s1">&#39;_CudaStreamBase&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_dummy_type</span><span class="p">(</span><span class="s1">&#39;CudaStreamBase&#39;</span><span class="p">)</span>
    <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s1">&#39;_CudaEventBase&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_dummy_type</span><span class="p">(</span><span class="s1">&#39;CudaEventBase&#39;</span><span class="p">)</span>


<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_lazy_new</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
    <span class="n">_lazy_init</span><span class="p">()</span>
    <span class="c1"># We may need to call lazy init again if we are a forked child</span>
    <span class="c1"># del _CudaBase.__new__</span>
    <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">_CudaBase</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="n">is_cuda</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="n">is_sparse</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="k">with</span> <span class="n">device</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_device</span><span class="p">()):</span>
            <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">type</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

    <span class="fm">__new__</span> <span class="o">=</span> <span class="n">_lazy_new</span>


<span class="k">class</span> <span class="nc">DoubleStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaDoubleStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">FloatStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaFloatStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">LongStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaLongStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">IntStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaIntStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">ShortStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaShortStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">CharStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaCharStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">ByteStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaByteStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">HalfStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaHalfStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">BoolStorage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaBoolStorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>


<span class="k">class</span> <span class="nc">BFloat16Storage</span><span class="p">(</span><span class="n">_CudaBase</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">CudaBFloat16StorageBase</span><span class="p">,</span> <span class="n">_StorageBase</span><span class="p">):</span>
    <span class="k">pass</span>

<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">DoubleStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">FloatStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">LongStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">IntStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ShortStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">CharStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ByteStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">HalfStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">BoolStorage</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_storage_classes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">BFloat16Storage</span><span class="p">)</span>

<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">sparse</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">profiler</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">nvtx</span>
<span class="kn">from</span> <span class="nn">.streams</span> <span class="kn">import</span> <span class="n">Stream</span><span class="p">,</span> <span class="n">Event</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">amp</span>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
         <script src="../../_static/jquery.js"></script>
         <script src="../../_static/underscore.js"></script>
         <script src="../../_static/doctools.js"></script>
         <script src="../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>